{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "9dbbf3cc-f7fd-4026-a02b-3f4ce878c145",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig, GenerationConfig, pipeline\n",
    "\n",
    "import os\n",
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "6067e62f-198f-4682-a987-0c9a51aa7e1e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bb61b82eb6ad452e90cf6df7be097690",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# This recipe requires a HuggingFace token to access the llama models from the hub. You will need to create a HuggingFace login\n",
    "# and generate a token for use at https://huggingface.co/settings/tokens. You will also need to visit the model card at \n",
    "# https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct and accept the terms to use the model. Once you have generated the token,\n",
    "# save it in an environment variable named HUGGINGFACE_TOKEN and read it in the recipe as shown in the snippet below.\n",
    "from huggingface_hub import login\n",
    "hf_token = os.environ.get('HUGGINGFACE_TOKEN')\n",
    "login(token=hf_token)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ab8e1f27-f260-4ff1-bf39-fbb3732028ac",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name = \"meta-llama/Meta-Llama-3.1-8B-Instruct\"\n",
    "quantization_config = BitsAndBytesConfig(load_in_4bit=True,\n",
    "                                         bnb_4bit_compute_dtype=torch.bfloat16,\n",
    "                                         bnb_4bit_use_double_quant=True,\n",
    "                                         bnb_4bit_quant_type= \"nf4\"\n",
    "                                         )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "fe5b3e8e-44f2-405a-990f-c7cd817f8ce7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "91a77850d5d64081a1bcd5c690093113",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/4 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "model = AutoModelForCausalLM.from_pretrained(\n",
    "                                        \tmodel_name, \n",
    "                                            device_map=\"auto\", \n",
    "                                            torch_dtype=torch.bfloat16, \n",
    "                                            quantization_config=quantization_config)\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "95560a72-745c-480f-b362-2d344f74547e",
   "metadata": {},
   "outputs": [],
   "source": [
    "pipe = pipeline(\"text-generation\", \n",
    "                model=model, \n",
    "                tokenizer=tokenizer, \n",
    "                max_new_tokens=256, \n",
    "                pad_token_id = tokenizer.eos_token_id,\n",
    "                eos_token_id=model.config.eos_token_id,\n",
    "                num_beams=4,\n",
    "                early_stopping=True,\n",
    "                repetition_penalty=1.4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "1fb355ba-9efc-4c94-89c9-086d3caeb217",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt = [\n",
    "    {\"role\": \"user\", \"content\": \"What is your favourite country?\"},\n",
    "    {\"role\": \"assistant\", \"content\": \"Well, I am quite fascinated with Peru.\"},\n",
    "    {\"role\": \"user\", \"content\": \"What can you tell me about Peru?\"}\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "b06fb20a-9578-4c1e-a4bb-95310d7b0f47",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Peru! A country with a rich history, diverse culture, and breathtaking landscapes. Here are some interesting facts about Peru:\n",
      "\n",
      "1. **Location**: Peru is located in western South America, bordering the Pacific Ocean to the west, Ecuador and Colombia to the north, Brazil and Bolivia to the east, and Chile to the south.\n",
      "2. **History**: Peru has a long and complex history, with various civilizations rising and falling over the centuries. The Inca Empire, which flourished from the 13th to the 16th century, is one of the most famous and influential empires in Peruvian history.\n",
      "3. **Machu Picchu**: One of the Seven Wonders of the World, Machu Picchu is an Inca citadel located on a mountain ridge above the Urubamba Valley. It's a must-visit destination for any traveler to Peru.\n",
      "4. **Food**: Peruvian cuisine is a fusion of indigenous, Spanish, African, and Asian influences. Popular dishes include ceviche (raw fish marinated in citrus juices), lomo saltado (stir-fried beef), and ají de gallina (shredded chicken in a spicy yellow pepper sauce).\n",
      "5. **Language**: The official language is Spanish, but many\n"
     ]
    }
   ],
   "source": [
    "outputs = pipe(\n",
    "    prompt,\n",
    "    max_new_tokens=256,\n",
    ")\n",
    "print(outputs[0][\"generated_text\"][-1]['content'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "26214251-5fc1-4dd2-9fec-84c3d838c074",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt = [\n",
    "    {\"role\": \"user\", \"content\": \"Mary is twice as old as Sarah presently. Sarah is 6 years old.?\"},\n",
    "    {\"role\": \"assistant\", \"content\": \"Well, what can I help you with?\"},\n",
    "    {\"role\": \"user\", \"content\": \"Can you tell me in a step by step way on how old Mary will be after 5 years?\"}\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "d4d54b28-ebb6-48f0-9c5f-fa17e2b1a0d3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Here's the step-by-step solution:\n",
      "\n",
      "**Step 1: Determine Sarah's current age**\n",
      "Sarah is 6 years old.\n",
      "\n",
      "**Step 2: Determine Mary's current age**\n",
      "Since Mary is twice as old as Sarah, and Sarah is 6 years old, we can multiply Sarah's age by 2 to find Mary's age:\n",
      "Mary's age = 2 x Sarah's age\n",
      "Mary's age = 2 x 6\n",
      "Mary's age = 12 years old\n",
      "\n",
      "**Step 3: Calculate Mary's age after 5 years**\n",
      "To find out how old Mary will be after 5 years, we add 5 to her current age:\n",
      "Mary's age after 5 years = Mary's current age + 5\n",
      "Mary's age after 5 years = 12 + 5\n",
      "Mary's age after 5 years = 17 years old\n",
      "\n",
      "Therefore, Mary will be 17 years old after 5 years.\n"
     ]
    }
   ],
   "source": [
    "outputs = pipe(\n",
    "    prompt,\n",
    "    max_new_tokens=256,\n",
    ")\n",
    "print(outputs[0][\"generated_text\"][-1]['content'])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "chapter10",
   "language": "python",
   "name": "chapter10"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
